import requests
import csv
import os
import json
from lxml import etree
import pandas as pd

MOVIE_TYPES=["剧情","喜剧","动作","爱情"]

class Spider:
    def __init__(self):
        self.movie_page_url="https://m.douban.com/rexxar/api/v2/movie/recommend?"
        self.movie_detail_url="https://movie.douban.com/subject/{}/"
        self.movie_types=MOVIE_TYPES
        self.page_progress={}
        self.headers={
            "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0",
            "referer":"https://movie.douban.com/explore"
        }
    def init(self):
        if not os.path.exists("movie_data.csv"):
            with open("movie_data.csv","w",newline="",encoding="utf-8") as f:
                writer=csv.writer(f)
                writer.writerow([
                    'id',
                    'title',
                    'year',
                    'directors',
                    'casts',
                    'rating',
                    'cover',
                    'country',
                    'summary',
                    'type',
                    'lang',
                    'release_data',
                    'time',
                    'url'
                ])
    def load_page_progress(self):
        if os.path.exists("page_progress.json"):
            with open("page_progress.json","r",encoding="utf-8") as f:
                self.page_progress=json.load(f)

    def save_page_progress(self):
        with open("page_progress.json","w",encoding="utf-8") as f:
            json.dump(self.page_progress,f,ensure_ascii=False)
    def get_movie_page(self,type_name,max_page=1):
        for page in range(self.page_progress.get(type_name,1),max_page+1):
            print(f'第{page}页:')
            params={"start":(page-1)*20,"tags":type_name}
            try:
                res=requests.get(self.movie_page_url,headers=self.headers,params=params).json()
                movie_list=res["items"]
                for m in movie_list:
                    if m["type"]=="movie":
                        self.process_movie(m)
                self.page_progress[type_name]=page+1
                self.save_page_progress()
            except Exception as e:
                print(f'处理{type_name}第{page}页失败：{e}')
                break

    def process_movie(self,movie):
        movie_data=[]
        movie_data.append(movie['id'])
        movie_data.append(movie['title'])
        movie_data.append(movie['year'])
        movie_detail_res=requests.get(self.movie_detail_url.format(movie['id']),headers=self.headers)
        path=etree.HTML(movie_detail_res.text)
        movie_data.append(",".join(path.xpath('//a[@rel="v:directedBy"]/text()'))) #导演
        movie_data.append(",".join(path.xpath('//a[@rel="v:starring"]/text()'))) #主演
        movie_data.append(path.xpath('//strong[@property="v:average"]/text()')[0]) #评分
        movie_data.append(path.xpath('//img[@rel="v:image"]/@src')[0]) #封面
        movie_data.append(path.xpath('//span[contains(text(),"制片国家")]/following-sibling::br[1]/preceding-sibling::text()[1]')[0])
        movie_data.append(path.xpath('//span[@property="v:summary"]/text()')[0].strip())
        movie_data.append(",".join(path.xpath('//div[@id="info"]/span[@property="v:genre"]/text()')))
        movie_data.append(path.xpath('//span[contains(text(),"语言:")]/following-sibling::br[1]/preceding-sibling::text()[1]')[0])
        movie_data.append(path.xpath('//span[@property="v:initialReleaseDate"]/text()')[0][:10])
        movie_data.append(path.xpath('//span[@property="v:runtime"]/text()')[0])
        movie_data.append(self.movie_detail_url.format(movie["id"]))
        self.save_to_csv(movie_data)
    def save_to_csv(self,row):
        with open("movie_data.csv","a",newline="",encoding="utf-8") as f:
            writer=csv.writer(f)
            writer.writerow(row)

    def run(self):
        self.init()
        self.load_page_progress()
        for type in self.movie_types:
            print(f'获取分类:{type}')
            self.get_movie_page(type)


if __name__=="__main__":
    spider=Spider()
    spider.run()